package com.samp.solr.file2solr2;


import com.samp.solr.file2solr.ErrorIntentionExtract;
import com.samp.util.CnStringUtils;
import org.apache.commons.lang3.StringUtils;

import java.io.BufferedReader;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.DirectoryStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.*;

public class DataExtract {

    /**
     * 意图转换
     * @TODO 意图要分词。
     * @param rawIntention
     * @return
     */
    public static String transferIntention(String rawIntention){
        List<String> list = new ArrayList<>();
        String[] intentionSplits = rawIntention.trim().split("#");
        for( int i = 0 ;i < intentionSplits.length ; i++ ){
            String intention = intentionSplits[i];
            String[] details = intention.split("=");
//            System.out.println(rawIntention);
            if( details.length < 2 ){
                return null;
            }
            if(  !"default".equals(details[1]) && !"execute".equals(details[1])
                    && !"chat".equals(details[1]) && !"集外".equals(details[1]) ){
                list.add(ErrorIntentionExtract.tokenize(details[1]));
            }
        }
        return StringUtils.join(list.iterator(), " ");
    }

    private static void extract(String rawPath, String resultFilePath) throws Exception{
        Path filePath = Paths.get(rawPath);
        BufferedReader bufferReader = Files.newBufferedReader(filePath, StandardCharsets.UTF_8);
        String line;
        Map<String,String> clidMap = new HashMap<>();
        List<String> resultList = new ArrayList<>();
        try {
            int index = 1;
            bufferReader.readLine();
            long start = System.currentTimeMillis();
            while( (line = bufferReader.readLine()) != null ){
                String[] words = line.split(",");
                String clid = words[0].trim();
                String previousIntention = null;
                if( clidMap.containsKey(clid) ){
                    previousIntention = clidMap.get(clid);
                    //如果意图格式错误，直接contionue.
                }
                String simpleIntention = transferIntention(words[3]);
                if( StringUtils.isNotBlank(simpleIntention) ){
                    clidMap.put(clid,simpleIntention);
                }
                String text = words[2];
                String semantic = words[3];
                //如果原始只有一个字，先过滤掉。
                if( text.trim().length() == 1 ){
                    continue;
                }
                //对文本进行分词
                String tokenText = ErrorIntentionExtract.tokenize(text);
                //如果分词语料为结果为空，则去掉
                if( StringUtils.isBlank(tokenText) ){
                    continue;
                }
                //如果语料只有一个字，去掉
                if( tokenText.trim().length() == 1 ){
//                    System.out.println("skip .."+line);
                    continue;
                }
                //如果全是英文，也去掉。
                if(CnStringUtils.isEnglish(tokenText) ){
//                    System.out.println("skip ...."+line);
                    continue;
                }
                //通过意图，新增一条语料
                String newWords = ErrorIntentionExtract.intentionToWord(semantic);
                if( StringUtils.isNotBlank(newWords) ){
                    resultList.add(newWords+"\t"+ErrorIntentionExtract.removeLastSlot(semantic));
                }
                if( previousIntention != null ){
                    tokenText = appendPreviousIntention(tokenText ,previousIntention);
//                    tokenText = tokenText +"|"+previousIntention;
                }else {
                    //去掉第一轮数据，只保留多轮数据。
//                    continue;
                }
//                resultList.add(clid+"\t"+tokenText+"\t"+ErrorIntentionExtract.removeLastSlot(semantic));
                resultList.add(tokenText+"\t"+ErrorIntentionExtract.removeLastSlot(semantic));
                if( index % 10000 == 0 ){
                    long cost = System.currentTimeMillis()-start;
                    System.out.println("process 1000 lines.current index ="+index+", cost:"+cost);
                    start = System.currentTimeMillis();
                }
                index++;
            }
            ErrorIntentionExtract.saveResultToFile(resultList, resultFilePath);
            System.out.println("data process finished.");
        } catch (IOException e) {
            e.printStackTrace();
        }finally{
            bufferReader.close();
        }
        System.out.println("all done!");
    }

    public static String appendPreviousIntention(String tokenText, String previousIntention){
        String[] splits = previousIntention.split(" ");
        List<String> list = new ArrayList<>();
        for( String s : tokenText.split(" ") ){
            list.add(s);
        }
        for( String str: splits) {
            if( !list.contains(str) ){
                list.add(str);
            }
        }
        return StringUtils.join(list, " ");
    }
    public static void main(String[] args) {
//        ErrorIntentionExtract.loCustomDict();
//        ErrorIntentionExtract.loadStopWords();
//        String sourceDir = "D:\\gcc-omcp\\new\\zjk\\";
//        String distDir = "D:\\gcc-omcp\\new2\\trans_2_09\\";
//        Path dir = Paths.get(sourceDir);
////        tokenize("嗯是有人");
//        //通过Files.newDirectoryStream(dir)来遍历文件夹
//        try( DirectoryStream<Path> stream = Files.newDirectoryStream(dir) ){
//            for(Path path : stream){
//                if( !Files.isDirectory(path) ){
//                    System.out.println("process........"+path.getFileName());
//                    String rawPath = sourceDir+path.getFileName();
//                    String outPath = distDir+path.getFileName();
//                    extract(rawPath, outPath);
//                }
//            }
//        }catch(Exception e){
//            e.printStackTrace();
//        }
//
////        打印同义词使用情况
//        try {
//            String outDir = "D:\\gcc-omcp\\new2\\data\\";
//            ErrorIntentionExtract.printSynStatis(outDir+"008_synStatics.txt");
//        }catch (Exception e){
//            e.printStackTrace();
//        }

        try{
            String sourceDir = "D:\\gcc-omcp\\new2\\trans_2_09\\";
            String outDir = "D:\\gcc-omcp\\new2\\data\\";
            String outFileName = "009_total_words_error_all.txt";
            String rightFileName = "009_total_words_right_all.txt";
            String rightFileName2 = "009_total_words_right_all2.txt";
            String totalFile = "009_total_words.txt";
            ErrorIntentionExtract.staticWordsDistrib(sourceDir, outDir+outFileName,
                    outDir+rightFileName,outDir+rightFileName2, outDir+totalFile);
//            ErrorIntentionExtract.checkFile(outDir+totalFile);
        }catch (Exception e){
            e.printStackTrace();
        }

    }
}
